import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
#########################################################
from IPython.display import display
from IPython.core.display import HTML
import seaborn as sns
from time import time
################# --- Sci-kit Learn --- #################
from sklearn.preprocessing import StandardScaler,LabelEncoder
from sklearn.metrics import (
    accuracy_score,
    f1_score,
    classification_report,
    confusion_matrix,
    mean_absolute_error,
    mean_squared_error,
    r2_score
    )
from sklearn.ensemble import (
    RandomForestClassifier,
    BaggingClassifier,
    GradientBoostingClassifier,
    AdaBoostClassifier,
    HistGradientBoostingClassifier,
    VotingClassifier,
    StackingClassifier,
    
    RandomForestRegressor,
    GradientBoostingRegressor,
    BaggingRegressor,
    AdaBoostRegressor,
    StackingRegressor,
    VotingRegressor,
    HistGradientBoostingRegressor
    )
from sklearn.model_selection import (
    GridSearchCV,
    train_test_split,
    cross_val_score
    )
from sklearn.inspection import permutation_importance
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
from sklearn.linear_model import LogisticRegression,LinearRegression
from sklearn.svm import SVC
#########################################################
from xgboost import XGBClassifier,XGBRegressor, plot_importance
from tabpfn import TabPFNClassifier, TabPFNRegressor
#########################################################
from ucimlrepo import fetch_ucirepo
import warnings as kxe_kxe
import locale
#########################################################
locale.setlocale(locale.LC_NUMERIC, 'russian')
plt.rcParams['axes.formatter.use_locale'] = True
kxe_kxe.filterwarnings('ignore')


def get_permutation_importances(model, X_test, y_test):
    result = permutation_importance(model, X_test, y_test, n_repeats=10, random_state=42)
    return result.importances_mean

def one_rrstr(x, n=0): # округление до n знаков после запятой
    if n == 0:
        return str(x)
    fmt = '{:.' + str(n) + 'f}'
    return fmt.format(x).replace('.', ',')

def un_one_rrstr(x):
    return float(x.replace(',', '.'))

def un_rrstr(x):
    return np.vectorize(un_one_rrstr)(x)


def rrstr(x, n = 4):
    """
    Форматирует числа или массивы чисел с округлением до заданного(`n`) количества знаков после запятой.
    """
    rrstr1 = np.vectorize(one_rrstr)
    res = rrstr1(x, n)
    if res.size == 1:
        return str(res)
    return res

dataset = 'rice'

ident = {
    'iris' : 53,
    'rice' : 545
}

for name, idx in ident.items():
    print(f'"{name}" size: {fetch_ucirepo(id=idx).data.original.shape[0]}')

"iris" size: 150
"rice" size: 3810

BASE = fetch_ucirepo(id=ident[dataset])
target = BASE.data.targets.columns [0]
df = BASE.data.original

df.head()

display(HTML(BASE.variables[[i for i in BASE.variables.columns if i not in ['units', 'demographic','type']]].to_html(escape=False)))

(df[target].value_counts()/df[target].size * 100).transform(lambda x : rrstr(x,2) + ' %')

Class
Osmancik    57,22 %
Cammeo      42,78 %
Name: count, dtype: object

le = LabelEncoder()
df[target] = le.fit_transform(df[target])

label_dict = dict(list(zip(list(le.classes_), list(range(len(le.classes_))))))

for key, value in label_dict.items():
    print(f'{key:^{max(list(map(len,label_dict.keys())))}}  -  {value}')

 Cammeo   -  0
Osmancik  -  1

scaler = StandardScaler()
X = df.drop(target, axis=1)
y = df[target]
X_scaled = scaler.fit_transform(X)

X_train, X_test, y_train, y_test = train_test_split(
    X_scaled, y, test_size=0.2, stratify=y, random_state=42
)

print("Размер обучающей выборки:", X_train.shape)
print("Размер тестовой выборки:", X_test.shape)

Размер обучающей выборки: (3048, 7)
Размер тестовой выборки: (762, 7)

sns.pairplot(df, hue=target)
plt.show()

df.hist(figsize=(10, 8))
plt.show()

plt.figure(figsize=(10, 8))
corr_matrix = np.corrcoef(X_scaled.T)
sns.heatmap(corr_matrix, annot=True, fmt=".2f", cmap="coolwarm")
plt.xticks(np.arange(7)+0.5, df.columns[:-1], rotation=45)
plt.yticks(np.arange(7)+0.5, df.columns[:-1], rotation=0)
plt.title("Корреляционная матрица числовых признаков")
plt.show()

def evaluate_model(model, X_train, y_train, X_test, y_test):
    """
    Оценка производительности модели машинного обучения на тестовых данных.
    
    Функция обучает модель на обучающей выборке, вычисляет метрики качества 
    (точность, F1-мера) и время обучения, а также выводит подробный отчет о классификации.
    
    Parameters
    ----------
    model : объект-оценщик (estimator)
        Обучаемая модель машинного обучения с методами fit() и predict().
    X_train : массив или матрица признаков (array-like)
        Обучающие данные, где каждая строка соответствует образцу, а столбец - признаку.
    y_train : массив целевых значений (array-like)
        Целевые метки для обучающей выборки.
    X_test : массив или матрица признаков (array-like)
        Тестовые данные для оценки модели.
    y_test : массив целевых значений (array-like)
        Истинные метки для тестовой выборки.
    
    Returns
    -------
    acc : float
        Точность модели на тестовой выборке (от 0 до 1).
    f1 : float
        Взвешенная F1-мера модели на тестовой выборке (от 0 до 1).
    train_time : float
        Время обучения модели в секундах.
    """
    start = time()
    model.fit(X_train, y_train)
    train_time = time() - start
    
    y_pred = model.predict(X_test)
    
    acc = accuracy_score(y_test, y_pred)
    f1 = f1_score(y_test, y_pred, average='weighted')
    
    print(f"Model: {model.__class__.__name__}")
    print(f"Accuracy: {acc:.4f}, F1-score: {f1:.4f}, Train time: {train_time:.4f}s")
    print(classification_report(y_test, y_pred))
    
    return acc, f1, train_time

results = dict()

param_grid = {
    'n_estimators': [50, 100, 150],
    'max_depth': [None, 5, 10],
    'min_samples_split': [2, 5, 10],
    'min_samples_leaf': [1, 2, 4]
}

rf = RandomForestClassifier(random_state=42)
grid_search = GridSearchCV(rf, param_grid, cv=5, scoring='accuracy', n_jobs=-1)
grid_search.fit(X_train, y_train)
best_rf = grid_search.best_estimator_

acc, f1, train_time = evaluate_model(best_rf, X_train, y_train, X_test, y_test)

results['Random Forest'] = (acc, f1, train_time)

print('Лучшие параметры:\n')
for key, value in grid_search.best_params_.items():
    print(f'{key} = {value}')

Model: RandomForestClassifier
Accuracy: 0.9186, F1-score: 0.9183, Train time: 0.4766s
              precision    recall  f1-score   support

           0       0.93      0.88      0.90       326
           1       0.91      0.95      0.93       436

    accuracy                           0.92       762
   macro avg       0.92      0.91      0.92       762
weighted avg       0.92      0.92      0.92       762

Лучшие параметры:

max_depth = 5
min_samples_leaf = 1
min_samples_split = 5
n_estimators = 150

importances = best_rf.feature_importances_
plt.barh(df.columns[:-1], importances)
plt.xlabel('Importance')
plt.title('Feature Importance (Random Forest)')
plt.show()

param_grid = {
    'learning_rate': [0.01, 0.1, 0.2],
    'n_estimators': [50, 100, 150],
    'max_depth': [3, 5, 7],
    'min_samples_split': [2, 5, 10]
}

gb = GradientBoostingClassifier(random_state=42)
grid_search = GridSearchCV(gb, param_grid, cv=5, scoring='accuracy', n_jobs=-1)
grid_search.fit(X_train, y_train)
best_gb = grid_search.best_estimator_

acc, f1, train_time = evaluate_model(best_gb, X_train, y_train, X_test, y_test)
results['Gradient Boosting'] = (acc, f1, train_time)
print('Лучшие параметры:\n')
for key, value in grid_search.best_params_.items():
    print(f'{key} = {value}')

Model: GradientBoostingClassifier
Accuracy: 0.9160, F1-score: 0.9156, Train time: 0.9517s
              precision    recall  f1-score   support

           0       0.93      0.87      0.90       326
           1       0.91      0.95      0.93       436

    accuracy                           0.92       762
   macro avg       0.92      0.91      0.91       762
weighted avg       0.92      0.92      0.92       762

Лучшие параметры:

learning_rate = 0.01
max_depth = 3
min_samples_split = 2
n_estimators = 150

importances = best_gb.feature_importances_
plt.barh(df.columns[:-1], importances)
plt.xlabel('Importance')
plt.title('Feature Importance (Gradient Boosting)')
plt.show()

base_model = DecisionTreeClassifier(random_state=42)

param_grid = {
    'n_estimators': [10, 50, 100],
    'max_samples': [0.5, 0.7, 1.0],
    'max_features': [0.5, 0.7, 1.0],
    'bootstrap': [True, False]
}

bagging = BaggingClassifier(estimator=base_model, random_state=42)
grid_search = GridSearchCV(bagging, param_grid, cv=5, scoring='accuracy', n_jobs=-1)
grid_search.fit(X_train, y_train)
best_bagging = grid_search.best_estimator_

acc, f1, train_time = evaluate_model(best_bagging, X_train, y_train, X_test, y_test)
results['Bagging'] = (acc, f1, train_time)
print('Лучшие параметры:\n')
for key, value in grid_search.best_params_.items():
    print(f'{key} = {value}')

Model: BaggingClassifier
Accuracy: 0.9147, F1-score: 0.9144, Train time: 0.5343s
              precision    recall  f1-score   support

           0       0.92      0.88      0.90       326
           1       0.91      0.94      0.93       436

    accuracy                           0.91       762
   macro avg       0.92      0.91      0.91       762
weighted avg       0.91      0.91      0.91       762

Лучшие параметры:

bootstrap = False
max_features = 0.7
max_samples = 0.5
n_estimators = 100

base_model = DecisionTreeClassifier(max_depth=1)  # Слабая модель (стамп)

param_grid = {
    'n_estimators': [50, 100, 150],
    'learning_rate': [0.01, 0.1, 1.0],
    'algorithm': ['SAMME', 'SAMME.R']
}

adaboost = AdaBoostClassifier(estimator=base_model, random_state=42)
grid_search = GridSearchCV(adaboost, param_grid, cv=5, scoring='accuracy', n_jobs=-1)
grid_search.fit(X_train, y_train)
best_adaboost = grid_search.best_estimator_

acc, f1, train_time = evaluate_model(best_adaboost, X_train, y_train, X_test, y_test)
results['AdaBoost'] = (acc, f1, train_time)

print('Лучшие параметры:\n')
for key, value in grid_search.best_params_.items():
    print(f'{key} = {value}')

Model: AdaBoostClassifier
Accuracy: 0.9160, F1-score: 0.9158, Train time: 0.3478s
              precision    recall  f1-score   support

           0       0.92      0.88      0.90       326
           1       0.91      0.94      0.93       436

    accuracy                           0.92       762
   macro avg       0.92      0.91      0.91       762
weighted avg       0.92      0.92      0.92       762

Лучшие параметры:

algorithm = SAMME
learning_rate = 0.1
n_estimators = 100

importances = best_adaboost.feature_importances_
plt.barh(df.columns[:-1], importances)
plt.xlabel('Importance')
plt.title('Feature Importance (AdaBoost)')
plt.show()

param_grid = {
    'n_estimators': [50, 100, 150],
    'max_depth': [3, 5, 7],
    'learning_rate': [0.01, 0.1, 0.2],
    'subsample': [0.8, 1.0],
    'colsample_bytree': [0.8, 1.0],
    'reg_alpha': [0, 0.1, 0.5],
    'reg_lambda': [0, 0.1, 0.5]
}

xgb = XGBClassifier(random_state=42, eval_metric='logloss')
grid_search = GridSearchCV(xgb, param_grid, cv=5, scoring='accuracy', n_jobs=-1)
grid_search.fit(X_train, y_train)
best_xgb = grid_search.best_estimator_

acc, f1, train_time = evaluate_model(best_xgb, X_train, y_train, X_test, y_test)
results['XGBoost'] = (acc, f1, train_time)

print('Лучшие параметры:\n')
for key, value in grid_search.best_params_.items():
    print(f'{key} = {value}')

Model: XGBClassifier
Accuracy: 0.9147, F1-score: 0.9144, Train time: 0.0306s
              precision    recall  f1-score   support

           0       0.92      0.88      0.90       326
           1       0.91      0.94      0.93       436

    accuracy                           0.91       762
   macro avg       0.92      0.91      0.91       762
weighted avg       0.91      0.91      0.91       762

Лучшие параметры:

colsample_bytree = 1.0
learning_rate = 0.1
max_depth = 3
n_estimators = 50
reg_alpha = 0.5
reg_lambda = 0
subsample = 0.8

plot_importance(best_xgb, title='Feature Importance (XGBoost)', xlabel='F Score', ylabel='Features')
plt.show()

for i in range(7):
    print(f'f{i} = {df.columns[:-1][i]}')

f0 = Area
f1 = Perimeter
f2 = Major_Axis_Length
f3 = Minor_Axis_Length
f4 = Eccentricity
f5 = Convex_Area
f6 = Extent

importances = pd.DataFrame({
    'XGBoost': best_xgb.feature_importances_,
    'RandomForest': best_rf.feature_importances_,
    'AdaBoost': best_adaboost.feature_importances_
}, index=df.columns[:-1])
importances.plot(kind='barh', figsize=(10, 6))
plt.title('Feature Importance Comparison')
plt.show()

# Настройка Random Forest
rf_params = {
    'n_estimators': [50, 100],
    'max_depth': [3, 5],
    'min_samples_split': [2, 5]
}
rf_grid = GridSearchCV(RandomForestClassifier(random_state=42), rf_params, cv=5, scoring='accuracy')
rf_grid.fit(X_train, y_train)

# Настройка Gradient Boosting
gb_params = {
    'learning_rate': [0.1, 0.2],
    'n_estimators': [50, 100],
    'max_depth': [3, 5]
}
gb_grid = GridSearchCV(GradientBoostingClassifier(random_state=42), gb_params, cv=5, scoring='accuracy')
gb_grid.fit(X_train, y_train)

GridSearchCV(cv=5, estimator=GradientBoostingClassifier(random_state=42),
             param_grid={'learning_rate': [0.1, 0.2], 'max_depth': [3, 5],
                         'n_estimators': [50, 100]},
             scoring='accuracy')

GridSearchCV(cv=5, estimator=GradientBoostingClassifier(random_state=42),
             param_grid={'learning_rate': [0.1, 0.2], 'max_depth': [3, 5],
                         'n_estimators': [50, 100]},
             scoring='accuracy')

GradientBoostingClassifier(n_estimators=50, random_state=42)

GradientBoostingClassifier(n_estimators=50, random_state=42)

estimators = [
    ('rf', rf_grid.best_estimator_),
    ('gb', gb_grid.best_estimator_),
    ('lr', LogisticRegression())
]

stacking = StackingClassifier(
    estimators=estimators,
    final_estimator=LogisticRegression(),
    cv=5
)
stacking.fit(X_train, y_train)

StackingClassifier(cv=5,
                   estimators=[('rf',
                                RandomForestClassifier(max_depth=5,
                                                       min_samples_split=5,
                                                       random_state=42)),
                               ('gb',
                                GradientBoostingClassifier(n_estimators=50,
                                                           random_state=42)),
                               ('lr', LogisticRegression())],
                   final_estimator=LogisticRegression())

StackingClassifier(cv=5,
                   estimators=[('rf',
                                RandomForestClassifier(max_depth=5,
                                                       min_samples_split=5,
                                                       random_state=42)),
                               ('gb',
                                GradientBoostingClassifier(n_estimators=50,
                                                           random_state=42)),
                               ('lr', LogisticRegression())],
                   final_estimator=LogisticRegression())

RandomForestClassifier(max_depth=5, min_samples_split=5, random_state=42)

GradientBoostingClassifier(n_estimators=50, random_state=42)

LogisticRegression()

LogisticRegression()

acc, f1, train_time = evaluate_model(stacking, X_train, y_train, X_test, y_test)
results['Stacking'] = (acc, f1, train_time)

print('Лучшие параметры:\n')
for key, value in gb_grid.best_params_.items():
    print(f'{key} = {value}')

Model: StackingClassifier
Accuracy: 0.9134, F1-score: 0.9131, Train time: 3.2294s
              precision    recall  f1-score   support

           0       0.92      0.87      0.90       326
           1       0.91      0.94      0.93       436

    accuracy                           0.91       762
   macro avg       0.91      0.91      0.91       762
weighted avg       0.91      0.91      0.91       762

Лучшие параметры:

learning_rate = 0.1
max_depth = 3
n_estimators = 50

lr_params = {
    'C': [0.1, 1, 10],  # Обратный коэффициент регуляризации
    'penalty': ['l1', 'l2'],
    'solver': ['liblinear']
}
lr_grid = GridSearchCV(LogisticRegression(), lr_params, cv=5, scoring='accuracy')
lr_grid.fit(X_train, y_train)

GridSearchCV(cv=5, estimator=LogisticRegression(),
             param_grid={'C': [0.1, 1, 10], 'penalty': ['l1', 'l2'],
                         'solver': ['liblinear']},
             scoring='accuracy')

GridSearchCV(cv=5, estimator=LogisticRegression(),
             param_grid={'C': [0.1, 1, 10], 'penalty': ['l1', 'l2'],
                         'solver': ['liblinear']},
             scoring='accuracy')

LogisticRegression(C=10, solver='liblinear')

LogisticRegression(C=10, solver='liblinear')

voting_clf = VotingClassifier(
    estimators=[
        ('lr', lr_grid.best_estimator_),
        ('rf', best_rf),
        ('gb', best_gb)
    ],
    voting='soft'  # Используем усреднение вероятностей
)
voting_clf.fit(X_train, y_train)

VotingClassifier(estimators=[('lr',
                              LogisticRegression(C=10, solver='liblinear')),
                             ('rf',
                              RandomForestClassifier(max_depth=5,
                                                     min_samples_split=5,
                                                     n_estimators=150,
                                                     random_state=42)),
                             ('gb',
                              GradientBoostingClassifier(learning_rate=0.01,
                                                         n_estimators=150,
                                                         random_state=42))],
                 voting='soft')

VotingClassifier(estimators=[('lr',
                              LogisticRegression(C=10, solver='liblinear')),
                             ('rf',
                              RandomForestClassifier(max_depth=5,
                                                     min_samples_split=5,
                                                     n_estimators=150,
                                                     random_state=42)),
                             ('gb',
                              GradientBoostingClassifier(learning_rate=0.01,
                                                         n_estimators=150,
                                                         random_state=42))],
                 voting='soft')

LogisticRegression(C=10, solver='liblinear')

RandomForestClassifier(max_depth=5, min_samples_split=5, n_estimators=150,
                       random_state=42)

GradientBoostingClassifier(learning_rate=0.01, n_estimators=150,
                           random_state=42)

acc, f1, train_time = evaluate_model(voting_clf, X_train, y_train, X_test, y_test)
results['Voting'] = (acc, f1, train_time)

print('Лучшие параметры:\n')
for key, value in lr_grid.best_params_.items():
    print(f'{key} = {value}')

Model: VotingClassifier
Accuracy: 0.9160, F1-score: 0.9157, Train time: 1.5380s
              precision    recall  f1-score   support

           0       0.92      0.88      0.90       326
           1       0.91      0.94      0.93       436

    accuracy                           0.92       762
   macro avg       0.92      0.91      0.91       762
weighted avg       0.92      0.92      0.92       762

Лучшие параметры:

C = 10
penalty = l2
solver = liblinear

importances = pd.DataFrame({
    'Logistic Regression': lr_grid.best_estimator_.coef_[0],
    'Random Forest': best_rf.feature_importances_,
    'Gradient Boosting': best_gb.feature_importances_
}, index=df.columns[:-1])
importances.plot(kind='barh', figsize=(10, 6))
plt.title('Feature Importance Comparison in Voting')
plt.show()

param_grid = {
    'learning_rate': [0.01, 0.1, 0.2],
    'max_iter': [100, 200, 300],
    'max_depth': [3, 5, 7],
    'l2_regularization': [0, 0.1, 0.5],
    'min_samples_leaf': [1, 5, 10, 20]
}

hist_gb = HistGradientBoostingClassifier(random_state=42)
grid_search = GridSearchCV(hist_gb, param_grid, cv=5, scoring='accuracy', n_jobs=-1)
grid_search.fit(X_train, y_train)
best_hist_gb = grid_search.best_estimator_

acc, f1, train_time = evaluate_model(best_hist_gb, X_train, y_train, X_test, y_test)
results['HistGradientBoosting'] = (acc, f1, train_time)

print('Лучшие параметры:\n')
for key, value in grid_search.best_params_.items():
    print(f'{key} = {value}')

Model: HistGradientBoostingClassifier
Accuracy: 0.9160, F1-score: 0.9158, Train time: 0.3945s
              precision    recall  f1-score   support

           0       0.92      0.88      0.90       326
           1       0.91      0.94      0.93       436

    accuracy                           0.92       762
   macro avg       0.92      0.91      0.91       762
weighted avg       0.92      0.92      0.92       762

Лучшие параметры:

l2_regularization = 0.5
learning_rate = 0.01
max_depth = 3
max_iter = 300
min_samples_leaf = 20

results_df = pd.DataFrame(results).T
results_df.columns = ['Accuracy', 'F1-score', 'Train time (s)']
for i in results_df.columns:
    results_df[i] = rrstr(results_df[i].values,4)

results_df

models = results_df.sort_values('Train time (s)').index
train_times = results_df.sort_values('Train time (s)')['Train time (s)'].values

plt.figure(figsize=(15, 6))
plt.bar(models, train_times)
plt.ylabel('Train Time (s)')
plt.title('Время обучения моделей')
plt.show()

# Рассчитываем важность признакей для всех моделей
def get_permutation_importances(model, X_test, y_test):
    result = permutation_importance(model, X_test, y_test, n_repeats=10, random_state=42)
    return result.importances_mean

# Словарь моделей
models_dict = {
    'Random Forest': best_rf,
    'Gradient Boosting': best_gb,
    'Bagging': best_bagging,
    'AdaBoost': best_adaboost,
    'XGBoost': best_xgb,
    'Stacking': stacking,
    'Voting': voting_clf,
    'HistGradientBoosting': best_hist_gb
}

# Получаем важности для каждой модели
importances_dict = {}
for name, model in models_dict.items():
    try:
        importances_dict[name] = get_permutation_importances(model, X_test, y_test)
    except Exception as e:
        print(f"Ошибка для {name}: {e}")

# Создаем DataFrame
importances_df = pd.DataFrame(importances_dict, index=df.columns[:-1])

# Визуализация
plt.figure(figsize=(14, 8))
importances_df.plot(kind='barh', ax=plt.gca())
plt.title('Сравнение важности признакей для всех моделей (permutation importance)')
plt.xlabel('Важность признака')
plt.ylabel('Признаки')
plt.legend(title='Модели', bbox_to_anchor=(1.05, 1), loc='upper left')
plt.tight_layout()
plt.show()

cv_scores = {}
for name, model in models_dict.items():
    scores_accuracy = cross_val_score(model, X_scaled, y, cv=5, scoring='accuracy').mean()
    scores_f1_macro = cross_val_score(model, X_scaled, y, cv=5, scoring='f1_macro').mean()

    cv_scores[name] = (scores_accuracy, scores_f1_macro)


differencer = lambda x: '+' + rrstr(x,5) if x>0 else '-' + rrstr(x,5)


cv_scores = pd.DataFrame(cv_scores).T
cv_scores.columns = ['Accuracy', 'F1-score']

cv_scores['Difference Accuracy'] = (cv_scores.Accuracy - results_df.Accuracy.transform(un_rrstr)).transform (differencer)
cv_scores['Difference F1-score'] = (cv_scores['F1-score'] - results_df['F1-score'].transform(un_rrstr)).transform (differencer)


cv_scores['NonCV Accuracy'] = results_df['Accuracy']
cv_scores['NonCV F1-score'] = results_df['F1-score']

cv_scores['Accuracy'] = cv_scores['Accuracy'].transform(lambda x: rrstr(x,4))
cv_scores['F1-score'] = cv_scores['F1-score'].transform(lambda x: rrstr(x,4))

cv_scores

#  Загрузка модели
model = TabPFNClassifier(n_estimators=33)

# Прогнозирование
model.fit(X_train, y_train)
y_pred = model.predict(X_train)

acc, f1, train_time = evaluate_model(model, X_train, y_train, X_test, y_test)
results['TabPFN'] = (acc, f1, train_time)

Model: TabPFNClassifier
Accuracy: 0.9160, F1-score: 0.9157, Train time: 0.6085s
              precision    recall  f1-score   support

           0       0.92      0.88      0.90       326
           1       0.91      0.94      0.93       436

    accuracy                           0.92       762
   macro avg       0.92      0.91      0.91       762
weighted avg       0.92      0.92      0.92       762

results_df = pd.DataFrame(results).T
results_df.columns = ['Accuracy', 'F1-score', 'Train time (s)']
for i in results_df.columns:
    results_df[i] = rrstr(results_df[i].values,4)

results_df

scores_accuracy_tabpfn = cross_val_score(model, X_scaled, y, cv=5, scoring='accuracy').mean()
scores_f1_macro_tabpfn = cross_val_score(model, X_scaled, y, cv=5, scoring='f1_macro').mean()

arrays = (rrstr(np.array([scores_accuracy, scores_f1_macro]),4),np.array([differencer(scores_accuracy - acc), differencer(scores_f1_macro-f1)]),rrstr(np.array([acc, f1]),4) )
arrays = np.hstack(arrays)
cv_scores.loc['TabPFN'] = arrays

cv_scores

# Рассчитываем важность признаков для всех моделей
def get_permutation_importances(model, X_test, y_test):
    result = permutation_importance(model, X_test, y_test, n_repeats=10, random_state=42)
    return result.importances_mean

# Словарь моделей
models_dict = {
    'TabPFN':model
}

# Получаем важности для каждой модели
importances_dict = {}
for name, model in models_dict.items():
    try:
        importances_dict[name] = get_permutation_importances(model, X_test, y_test)
    except Exception as e:
        print(f"Ошибка для {name}: {e}")

# Создаем DataFrame
importances_df = pd.DataFrame(importances_dict, index=df.columns[:-1])

# Визуализация
plt.figure(figsize=(8, 5))
importances_df.plot(kind='barh', ax=plt.gca())
plt.title('feature importances для TabPFN')
plt.xlabel('Важность признака')
plt.ylabel('Признаки')
plt.tight_layout()
plt.show()

ident={
    'Parkinsons' : 189,
    'Strength':165,
    'Bikes':560,
}

for name, idx in ident.items():
    print(f'"{name}" shape: {fetch_ucirepo(id=idx).data.original.shape}')

"Parkinsons" shape: (5875, 22)
"Strength" shape: (1030, 9)
"Bikes" shape: (8760, 14)

dataset = 'Parkinsons'

BASE = fetch_ucirepo(id=ident[dataset])
df = BASE.data.original
target = BASE.data.targets.columns
target

Index(['motor_UPDRS', 'total_UPDRS'], dtype='object')

df.head()

df.drop('subject#',axis=1, inplace=True)
df

df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 5875 entries, 0 to 5874
Data columns (total 21 columns):
 #   Column         Non-Null Count  Dtype  
---  ------         --------------  -----  
 0   age            5875 non-null   int64  
 1   sex            5875 non-null   int64  
 2   test_time      5875 non-null   float64
 3   Jitter(%)      5875 non-null   float64
 4   Jitter(Abs)    5875 non-null   float64
 5   Jitter:RAP     5875 non-null   float64
 6   Jitter:PPQ5    5875 non-null   float64
 7   Jitter:DDP     5875 non-null   float64
 8   Shimmer        5875 non-null   float64
 9   Shimmer(dB)    5875 non-null   float64
 10  Shimmer:APQ3   5875 non-null   float64
 11  Shimmer:APQ5   5875 non-null   float64
 12  Shimmer:APQ11  5875 non-null   float64
 13  Shimmer:DDA    5875 non-null   float64
 14  NHR            5875 non-null   float64
 15  HNR            5875 non-null   float64
 16  RPDE           5875 non-null   float64
 17  DFA            5875 non-null   float64
 18  PPE            5875 non-null   float64
 19  motor_UPDRS    5875 non-null   float64
 20  total_UPDRS    5875 non-null   float64
dtypes: float64(19), int64(2)
memory usage: 964.0 KB

df.describe()

df.isna().sum()

age              0
sex              0
test_time        0
Jitter(%)        0
Jitter(Abs)      0
Jitter:RAP       0
Jitter:PPQ5      0
Jitter:DDP       0
Shimmer          0
Shimmer(dB)      0
Shimmer:APQ3     0
Shimmer:APQ5     0
Shimmer:APQ11    0
Shimmer:DDA      0
NHR              0
HNR              0
RPDE             0
DFA              0
PPE              0
motor_UPDRS      0
total_UPDRS      0
dtype: int64

df.drop(target[0], axis=1, inplace=True)
target = target[1]
df

# Гистограмма с KDE
plt.figure(figsize=(12, 4))
plt.subplot(1, 2, 1)
sns.histplot(df['total_UPDRS'], kde=True, bins=30, color='skyblue')
plt.title('Распределение total_UPDRS')

# Ящик с усами
plt.subplot(1, 2, 2)
sns.boxplot(x=df['total_UPDRS'], color='lightgreen')
plt.title('Boxplot: total_UPDRS')
plt.tight_layout()
plt.show()

plt.figure(figsize=(16, 14))
corr_matrix = df.corr()
sns.heatmap(corr_matrix, annot=True, fmt='.2f', cmap='coolwarm', linewidths=0.5)
plt.title('Матрица корреляций (Parkinsons dataset)')
plt.show()

def evaluate_regression(model, X_train, y_train, X_test, y_test):
    start = time()
    model.fit(X_train, y_train)
    train_time = time() - start
    y_pred = model.predict(X_test)
    mae = mean_absolute_error(y_test, y_pred)
    rmse = mean_squared_error(y_test, y_pred, squared=False)
    r2 = r2_score(y_test, y_pred)
    print(f"Model: {model.__class__.__name__}")
    print(f"MAE: {mae:.4f}, RMSE: {rmse:.4f}, R²: {r2:.4f}, Train time: {train_time:.4f}s")
    return mae, rmse, r2, train_time

X = df.drop(target, axis=1)
y = df[target]

# Масштабирование признаков
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

# Разделение
X_train, X_test, y_train, y_test = train_test_split(
    X_scaled, y, test_size=0.2, random_state=42
)

results_regression = dict()

param_grid = {
    'n_estimators': [50, 100, 150],
    'max_depth': [None, 5, 10],
    'min_samples_split': [2, 5, 10],
    'min_samples_leaf': [1, 2, 4]
}
rf = RandomForestRegressor(random_state=42)
grid_search = GridSearchCV(rf, param_grid, cv=5, scoring='neg_mean_squared_error', n_jobs=-1)
grid_search.fit(X_train, y_train)
best_rf = grid_search.best_estimator_

mae, rmse, r2, train_time = evaluate_regression(best_rf, X_train, y_train, X_test, y_test)
results_regression['RandomForestRegressor'] = (mae, rmse, r2, train_time)

print('\nЛучшие параметры:\n')
for key, value in grid_search.best_params_.items():
    print(f'{key} = {value}')

Model: RandomForestRegressor
MAE: 0.7196, RMSE: 1.5894, R²: 0.9772, Train time: 6.8635s

Лучшие параметры:

max_depth = None
min_samples_leaf = 1
min_samples_split = 2
n_estimators = 150

importances = best_rf.feature_importances_
plt.barh(X.columns[np.argsort(importances)], importances[np.argsort(importances)])
plt.xlabel('Importance')
plt.title('Feature Importance (Random Forest)')
plt.show()

param_grid = {
    'learning_rate': [0.01, 0.1, 0.2],
    'n_estimators': [50, 100, 150],
    'max_depth': [3, 5, 7],
    'min_samples_split': [2, 5, 10]
}
gb = GradientBoostingRegressor(random_state=42)
grid_search = GridSearchCV(gb, param_grid, cv=5, scoring='neg_mean_squared_error', n_jobs=-1)
grid_search.fit(X_train, y_train)
best_gb = grid_search.best_estimator_

mae, rmse, r2, train_time = evaluate_regression(best_gb, X_train, y_train, X_test, y_test)
results_regression['GradientBoostingRegressor'] = (mae, rmse, r2, train_time)

print('\nЛучшие параметры:\n')
for key, value in grid_search.best_params_.items():
    print(f'{key} = {value}')

Model: GradientBoostingRegressor
MAE: 1.0351, RMSE: 1.8023, R²: 0.9707, Train time: 6.6971s

Лучшие параметры:

learning_rate = 0.1
max_depth = 7
min_samples_split = 5
n_estimators = 150

importances = best_gb.feature_importances_
plt.barh(df.columns[:-1][np.argsort(importances)], importances[np.argsort(importances)])
plt.xlabel('Importance')
plt.title('Feature Importance (Gradient Boosting)')
plt.show()

# Базовая модель - дерево решений
base_model = DecisionTreeRegressor(random_state=42)

# Параметры для настройки
param_grid = {
    'n_estimators': [50, 100, 150],
    'max_samples': [0.5, 0.7, 1.0],
    'max_features': [0.5, 0.7, 1.0],
    'bootstrap': [True, False]
}

# Настройка гиперпараметров
bagging = BaggingRegressor(estimator=base_model, random_state=42)
grid_search = GridSearchCV(bagging, param_grid, cv=5, scoring='neg_mean_squared_error', n_jobs=-1)
grid_search.fit(X_train, y_train)
best_bagging = grid_search.best_estimator_

mae, rmse, r2, train_time = evaluate_regression(
    best_bagging, 
    X_train, 
    y_train, 
    X_test, 
    y_test
)

results_regression['BaggingRegressor'] = (mae, rmse, r2, train_time)

print('\nЛучшие параметры:\n')
for key, value in grid_search.best_params_.items():
    print(f'{key} = {value}')

Model: BaggingRegressor
MAE: 0.7174, RMSE: 1.5850, R²: 0.9773, Train time: 7.1530s

Лучшие параметры:

bootstrap = True
max_features = 1.0
max_samples = 1.0
n_estimators = 150

# График предсказанных vs реальных значений
plt.scatter(y_test, best_bagging.predict(X_test), alpha=0.7)
plt.plot([y.min(), y.max()], [y.min(), y.max()], 'r--')  # Линия идеального прогноза
plt.xlabel('True Values')
plt.ylabel('Predictions')
plt.title('Bagging: True vs Predicted')
plt.show()

importances = get_permutation_importances(best_bagging.estimators_[0], X_test, y_test)
plt.barh(df.columns[:-1][np.argsort(importances)], importances[np.argsort(importances)])
plt.xlabel('Importance')
plt.title('Feature Importance (Bagging)')
plt.show()

importances_comparison = pd.DataFrame({
    'Bagging': importances/max(importances+1),
    'Random Forest': best_rf.feature_importances_,
    'Gradient Boosting': best_gb.feature_importances_
}, index=df.columns[:-1])

importances_comparison.plot(kind='barh', figsize=(10, 8))
plt.title('Feature Importance Comparison')
plt.xlabel('Importance')
plt.ylabel('Features')
plt.show()

temp_res = pd.DataFrame(results_regression).T
temp_res.columns = ['MAE', 'RMSE', 'R2', 'time']
temp_res

# Базовая модель - дерево решений малой глубины
base_model = DecisionTreeRegressor(max_depth=1)

# Параметры для настройки
param_grid = {
    'n_estimators': [10, 30, 50, 100, 150, 200],
    'learning_rate': [0.01, 0.1, 0.3, 0.5, 0.8, 1.0],
    'loss': ['linear', 'square', 'exponential']
}

# Настройка гиперпараметров
adaboost = AdaBoostRegressor(estimator=base_model, random_state=42)
grid_search = GridSearchCV(adaboost, param_grid, cv=5, scoring='neg_mean_squared_error', n_jobs=-1)
grid_search.fit(X_train, y_train)
best_adaboost = grid_search.best_estimator_

mae, rmse, r2, train_time = evaluate_regression(
    best_adaboost, 
    X_train, 
    y_train, 
    X_test, 
    y_test
)

results_regression['AdaBoostRegressor'] = (mae, rmse, r2, train_time)

print('\nЛучшие параметры:\n')
for key, value in grid_search.best_params_.items():
    print(f'{key} = {value}')

Model: AdaBoostRegressor
MAE: 8.2904, RMSE: 9.6651, R²: 0.1570, Train time: 0.1004s

Лучшие параметры:

learning_rate = 1.0
loss = square
n_estimators = 30

# График предсказанных vs реальных значений
plt.scatter(y_test, best_adaboost.predict(X_test), alpha=0.7)
plt.plot([y.min(), y.max()], [y.min(), y.max()], 'r--')  # Линия идеального прогноза
plt.xlabel('True Values')
plt.ylabel('Predictions')
plt.title('AdaBoost: True vs Predicted')
plt.show()

param_grid = {
    'n_estimators': [50, 100, 150],
    'max_depth': [3, 5, 7],
    'learning_rate': [0.01, 0.1, 0.2],
    'subsample': [0.8, 1.0],
    'colsample_bytree': [0.8, 1.0],
    'reg_alpha': [0, 0.1, 0.5],
    'reg_lambda': [0, 0.1, 0.5]
}

xgb = XGBRegressor(random_state=42, eval_metric='rmse')
grid_search = GridSearchCV(xgb, param_grid, cv=5, scoring='neg_mean_squared_error', n_jobs=-1)
grid_search.fit(X_train, y_train)
best_xgb = grid_search.best_estimator_

mae, rmse, r2, train_time = evaluate_regression(best_xgb, X_train, y_train, X_test, y_test)

results_regression['XGBRegressor'] = (mae, rmse, r2, train_time)

print('\nЛучшие параметры:\n')
for key, value in grid_search.best_params_.items():
    print(f'{key} = {value}')

Model: XGBRegressor
MAE: 1.0424, RMSE: 1.7563, R²: 0.9722, Train time: 0.3906s

Лучшие параметры:

colsample_bytree = 1.0
learning_rate = 0.1
max_depth = 7
n_estimators = 150
reg_alpha = 0.1
reg_lambda = 0
subsample = 0.8

plt.scatter(y_test, best_xgb.predict(X_test), alpha=0.7)
plt.plot([y.min(), y.max()], [y.min(), y.max()], 'r--')
plt.xlabel('True Values')
plt.ylabel('Predictions')
plt.title('XGBoost: True vs Predicted (Parkinsons)')
plt.show()

plot_importance(best_xgb, title='Feature Importance (XGBoost)', xlabel='F Score', ylabel='Features')
plt.show()

for i in range(df.columns[:-1].size):
    print(f'f{i} = {df.columns[:-1][i]}')

f0 = age
f1 = sex
f2 = test_time
f3 = Jitter(%)
f4 = Jitter(Abs)
f5 = Jitter:RAP
f6 = Jitter:PPQ5
f7 = Jitter:DDP
f8 = Shimmer
f9 = Shimmer(dB)
f10 = Shimmer:APQ3
f11 = Shimmer:APQ5
f12 = Shimmer:APQ11
f13 = Shimmer:DDA
f14 = NHR
f15 = HNR
f16 = RPDE
f17 = DFA
f18 = PPE

param_grid = {
    'n_estimators': [50, 100, 150],
    'max_depth': [None, 5, 10],
    'min_samples_split': [2, 5, 10],
    'min_samples_leaf': [1, 2, 4]
}
rf = RandomForestRegressor(random_state=42)
rf_grid = GridSearchCV(rf, param_grid, cv=5, scoring='neg_mean_squared_error', n_jobs=-1)
rf_grid.fit(X_train, y_train)
best_rf = rf_grid.best_estimator_

param_grid = {
    'learning_rate': [0.01, 0.1, 0.2],
    'n_estimators': [50, 100, 150],
    'max_depth': [3, 5, 7],
    'min_samples_split': [2, 5, 10]
}
gb = GradientBoostingRegressor(random_state=42)
gb_grid = GridSearchCV(gb, param_grid, cv=5, scoring='neg_mean_squared_error', n_jobs=-1)
gb_grid.fit(X_train, y_train)
best_gb = gb_grid.best_estimator_

estimators = [
    ('rf', best_rf),
    ('gb', best_gb),
    ('lr', LinearRegression())
]
stacking = StackingRegressor(
    estimators=estimators,
    final_estimator=LinearRegression(),
    cv=5
)
stacking.fit(X_train, y_train)

StackingRegressor(cv=5,
                  estimators=[('rf',
                               RandomForestRegressor(n_estimators=150,
                                                     random_state=42)),
                              ('gb',
                               GradientBoostingRegressor(max_depth=7,
                                                         min_samples_split=5,
                                                         n_estimators=150,
                                                         random_state=42)),
                              ('lr', LinearRegression())],
                  final_estimator=LinearRegression())

StackingRegressor(cv=5,
                  estimators=[('rf',
                               RandomForestRegressor(n_estimators=150,
                                                     random_state=42)),
                              ('gb',
                               GradientBoostingRegressor(max_depth=7,
                                                         min_samples_split=5,
                                                         n_estimators=150,
                                                         random_state=42)),
                              ('lr', LinearRegression())],
                  final_estimator=LinearRegression())

RandomForestRegressor(n_estimators=150, random_state=42)

GradientBoostingRegressor(max_depth=7, min_samples_split=5, n_estimators=150,
                          random_state=42)

LinearRegression()

LinearRegression()

mae, rmse, r2, train_time = evaluate_regression(stacking, X_train, y_train, X_test, y_test)

results_regression['StackingRegressor'] = (mae, rmse, r2, train_time)

Model: StackingRegressor
MAE: 0.7880, RMSE: 1.5414, R²: 0.9786, Train time: 91.4056s

temp_res = pd.DataFrame(results_regression).T
temp_res.columns = ['MAE', 'RMSE', 'R2', 'time']
temp_res

plt.scatter(y_test, stacking.predict(X_test), alpha=0.7)
plt.plot([y.min(), y.max()], [y.min(), y.max()], 'r--')
plt.xlabel('True Values')
plt.ylabel('Predictions')
plt.title('Stacking: True vs Predicted')
plt.show()

# Базовые модели
models = [
    ('rf', RandomForestRegressor(random_state=42)),
    ('lr', LinearRegression()),
    ('gb', GradientBoostingRegressor(random_state=42))
]

# Настройка гиперпараметров каждой модели
param_grids = {
    'rf': {
        'n_estimators': [150,200],
        'max_depth': [None, 7],
        'min_samples_split': [2, 5, 7]
    },
    'gb': {
        'learning_rate': [0.01, 0.1, 0.3],
        'n_estimators': [150, 200],
        'max_depth': [5, 7]
    }
}

# Настройка Random Forest
rf_grid = GridSearchCV(models[0][1], param_grids['rf'], cv=5, scoring='neg_mean_squared_error', n_jobs=-1)
rf_grid.fit(X_train, y_train)
best_rf = rf_grid.best_estimator_

# Настройка Gradient Boosting
gb_grid = GridSearchCV(models[2][1], param_grids['gb'], cv=5, scoring='neg_mean_squared_error', n_jobs=-1)
gb_grid.fit(X_train, y_train)
best_gb = gb_grid.best_estimator_

# Используем лучшие параметры для Random Forest и Gradient Boosting
voting_regressor = VotingRegressor(
    estimators=[
        ('rf', best_rf),
        ('lr', LinearRegression(n_jobs=-1)),
        ('gb', best_gb)
    ],
    n_jobs=-1
)
voting_regressor.fit(X_train, y_train)

VotingRegressor(estimators=[('rf',
                             RandomForestRegressor(n_estimators=150,
                                                   random_state=42)),
                            ('lr', LinearRegression(n_jobs=-1)),
                            ('gb',
                             GradientBoostingRegressor(max_depth=7,
                                                       n_estimators=200,
                                                       random_state=42))],
                n_jobs=-1)

VotingRegressor(estimators=[('rf',
                             RandomForestRegressor(n_estimators=150,
                                                   random_state=42)),
                            ('lr', LinearRegression(n_jobs=-1)),
                            ('gb',
                             GradientBoostingRegressor(max_depth=7,
                                                       n_estimators=200,
                                                       random_state=42))],
                n_jobs=-1)

RandomForestRegressor(n_estimators=150, random_state=42)

LinearRegression(n_jobs=-1)

GradientBoostingRegressor(max_depth=7, n_estimators=200, random_state=42)

mae, rmse, r2, train_time = evaluate_regression(
    voting_regressor, 
    X_train, 
    y_train, 
    X_test, 
    y_test
)

results_regression['VotingRegressor'] = (mae, rmse, r2, train_time)

Model: VotingRegressor
MAE: 3.0640, RMSE: 3.7446, R²: 0.8735, Train time: 9.7425s

plt.scatter(y_test, voting_regressor.predict(X_test), alpha=0.7)
plt.plot([y.min(), y.max()], [y.min(), y.max()], 'r--')  # Линия идеального прогноза
plt.xlabel('True Values')
plt.ylabel('Predictions')
plt.title('Voting Regressor: True vs Predicted')
plt.show()

param_grid = {
    'learning_rate': [0.01, 0.1, 0.2],
    'max_iter': [100, 200, 300],
    'max_depth': [3, 5, 7],
    'l2_regularization': [0, 0.1, 0.5],
    'min_samples_leaf': [1, 5, 10, 20]
}
hist_gb = HistGradientBoostingRegressor(random_state=42)
grid_search = GridSearchCV(hist_gb, param_grid, cv=5, scoring='neg_mean_squared_error', n_jobs=-1)
grid_search.fit(X_train, y_train)
best_hist_gb = grid_search.best_estimator_

mae, rmse, r2, train_time = evaluate_regression(
    best_hist_gb, 
    X_train, 
    y_train, 
    X_test, 
    y_test
)

results_regression['HistGradientBoostingRegressor']=(mae, rmse, r2, train_time)

Model: HistGradientBoostingRegressor
MAE: 1.2451, RMSE: 2.0134, R²: 0.9634, Train time: 1.1233s

tab_model = TabPFNRegressor(n_estimators=33)

# Прогнозирование
tab_model.fit(X_train, y_train)
y_pred = tab_model.predict(X_train)

mae, rmse, r2, train_time = evaluate_regression(tab_model, X_train, y_train, X_test, y_test)

results_regression['TabPFNRegressor'] = (mae, rmse, r2, train_time)

Model: TabPFNRegressor
MAE: 0.2060, RMSE: 0.7836, R²: 0.9945, Train time: 3.0418s

# weights = get_permutation_importances(model, X_test,y_test)

# plt.imshow(weights, cmap='viridis')
# plt.xlabel('Features')
# plt.ylabel('Attention Head')
# plt.title('Attention Weights (TabPFN)')
# plt.colorbar()
# plt.show()

results_regression_df = pd.DataFrame(results_regression).T
results_regression_df.columns = ['MAE', 'RMSE', 'R2', 'Train time (s)']
results_regression_df.sort_values('R2',inplace=True)
for i in results_regression_df.columns:
    results_regression_df[i] = rrstr(results_regression_df[i].values,4)

results_regression_df

models = results_regression_df.sort_values('Train time (s)').index
train_times = results_regression_df.sort_values('Train time (s)')['Train time (s)'].values

plt.figure(figsize=(19, 6))
plt.bar(models, train_times)
plt.ylabel('Train Time (s)')
plt.title('Время обучения моделей')
plt.tight_layout()
plt.show()

# Словарь моделей
models_dict = dict(zip(results_regression_df.index, [
    best_adaboost,
    voting_regressor,
    best_hist_gb,
    best_gb,
    best_xgb,
    best_rf,
    best_bagging,
    stacking,
    tab_model
]))



cv_scores = {}



for name, model in models_dict.items():
    print(f'{name} :')
    MAE = cross_val_score(model, X_scaled, y, cv=5, scoring='neg_mean_absolute_error').mean()
    print(f'MAE = {MAE}, ',end='')
    RMSE = cross_val_score(model, X_scaled, y, cv=5, scoring='neg_root_mean_squared_error').mean()
    print(f'RMSE = {RMSE}, ',end='')
    R2 = cross_val_score(model, X_scaled, y, cv=5, scoring='r2').mean()
    print(f'R2 = {R2}, ')


    cv_scores[name] = (MAE, RMSE, R2)

AdaBoostRegressor :
MAE = -9.257981862948617, RMSE = -10.735921120025873, R2 = -0.6030393432024467, 
VotingRegressor :
MAE = -9.72947167854635, RMSE = -11.497741893011405, R2 = -0.9229352029816271, 
HistGradientBoostingRegressor :
MAE = -10.656623958835898, RMSE = -12.91020564963065, R2 = -1.5686642503711723, 
GradientBoostingRegressor :
MAE = -10.672421445987993, RMSE = -12.8920381584935, R2 = -1.5230521357232796, 
XGBRegressor :
MAE = -10.413070216626043, RMSE = -12.563328198884467, R2 = -1.3647285256165727, 
RandomForestRegressor :
MAE = -10.565688833134754, RMSE = -12.893326942442865, R2 = -1.7136209061433756, 
BaggingRegressor :
MAE = -10.56000670377305, RMSE = -12.900333815616966, R2 = -1.7252043080545678, 
StackingRegressor :
MAE = -9.975628638833516, RMSE = -11.74438638361255, R2 = -0.9138661632316246, 
TabPFNRegressor :
MAE = -9.2708119156711, RMSE = -11.410196353998405, R2 = -0.9645779643846366,

Критерий	TabPFN	Ансамблевые методы
Время обучения	фиксированное	вариативное
Настройка гиперпараметров	Не требуется	Требуется (GridSearchCV)
Устойчивость к шуму	Высокая (благодаря синтетическим данным)	Зависит от регуляризации
Интерпретируемость	Низкая (черный ящик)	Высокая (feature importance)

	Area	Perimeter	Major_Axis_Length	Minor_Axis_Length	Eccentricity	Convex_Area	Extent	Class
0	15231	525.578979	229.749878	85.093788	0.928882	15617	0.572896	Cammeo
1	14656	494.311005	206.020065	91.730972	0.895405	15072	0.615436	Cammeo
2	14634	501.122009	214.106781	87.768288	0.912118	14954	0.693259	Cammeo
3	13176	458.342987	193.337387	87.448395	0.891861	13368	0.640669	Cammeo
4	14688	507.166992	211.743378	89.312454	0.906691	15262	0.646024	Cammeo

	Accuracy	F1-score	Train time (s)
Random Forest	0,9186	0,9183	0,4766
Gradient Boosting	0,9160	0,9156	0,9517
Bagging	0,9147	0,9144	0,5343
AdaBoost	0,9160	0,9158	0,3478
XGBoost	0,9147	0,9144	0,0306
Stacking	0,9134	0,9131	3,2294
Voting	0,9160	0,9157	1,5380
HistGradientBoosting	0,9160	0,9158	0,3945

	Accuracy	F1-score	Difference Accuracy	Difference F1-score	NonCV Accuracy	NonCV F1-score
Random Forest	0,9268	0,9250	+0,00817	+0,00673	0,9186	0,9183
Gradient Boosting	0,9252	0,9233	+0,00920	+0,00768	0,9160	0,9156
Bagging	0,9257	0,9240	+0,01102	+0,00960	0,9147	0,9144
AdaBoost	0,9241	0,9224	+0,00815	+0,00658	0,9160	0,9158
XGBoost	0,9265	0,9248	+0,01181	+0,01039	0,9147	0,9144
Stacking	0,9273	0,9256	+0,01390	+0,01252	0,9134	0,9131
Voting	0,9278	0,9261	+0,01182	+0,01041	0,9160	0,9157
HistGradientBoosting	0,9265	0,9247	+0,01051	+0,00892	0,9160	0,9158

	Accuracy	F1-score	Train time (s)
Random Forest	0,9186	0,9183	0,4766
Gradient Boosting	0,9160	0,9156	0,9517
Bagging	0,9147	0,9144	0,5343
AdaBoost	0,9160	0,9158	0,3478
XGBoost	0,9147	0,9144	0,0306
Stacking	0,9134	0,9131	3,2294
Voting	0,9160	0,9157	1,5380
HistGradientBoosting	0,9160	0,9158	0,3945
TabPFN	0,9160	0,9157	0,6085

	name	role	description	missing_values
0	Area	Feature	Returns the number of pixels within the boundaries of the rice grain	no
1	Perimeter	Feature	Calculates the circumference by calculating the distance between pixels around the boundaries of the rice grain	no
2	Major_Axis_Length	Feature	The longest line that can be drawn on the rice grain, i.e. the main axis distance, gives	no
3	Minor_Axis_Length	Feature	The shortest line that can be drawn on the rice grain, i.e. the small axis distance, gives	no
4	Eccentricity	Feature	It measures how round the ellipse, which has the same moments as the rice grain, is	no
5	Convex_Area	Feature	Returns the pixel count of the smallest convex shell of the region formed by the rice grain	no
6	Extent	Feature	Returns the ratio of the region formed by the rice grain to the bounding box	no
7	Class	Target	Cammeo and Osmancik	no

	subject#	age	test_time	Jitter(%)	Jitter(Abs)	Jitter:RAP	Jitter:PPQ5	Jitter:DDP	Shimmer	...	Shimmer:APQ5	Shimmer:APQ11	Shimmer:DDA	NHR	HNR	RPDE	DFA	PPE	motor_UPDRS	total_UPDRS
0	1	72	5.6431	0.00662	0.000034	0.00401	0.00317	0.01204	0.02565	...	0.01309	0.01662	0.04314	0.014290	21.640	0.41888	0.54842	0.16006	28.199	34.398
1	1	72	12.6660	0.00300	0.000017	0.00132	0.00150	0.00395	0.02024	...	0.01072	0.01689	0.02982	0.011112	27.183	0.43493	0.56477	0.10810	28.447	34.894
2	1	72	19.6810	0.00481	0.000025	0.00205	0.00208	0.00616	0.01675	...	0.00844	0.01458	0.02202	0.020220	23.047	0.46222	0.54405	0.21014	28.695	35.389
3	1	72	25.6470	0.00528	0.000027	0.00191	0.00264	0.00573	0.02309	...	0.01265	0.01963	0.03317	0.027837	24.445	0.48730	0.57794	0.33277	28.905	35.810
4	1	72	33.6420	0.00335	0.000020	0.00093	0.00130	0.00278	0.01703	...	0.00929	0.01819	0.02036	0.011625	26.126	0.47188	0.56122	0.19361	29.187	36.375

	age	sex	test_time	Jitter(%)	Jitter(Abs)	Jitter:RAP	Jitter:PPQ5	Jitter:DDP	Shimmer	Shimmer(dB)	...	Shimmer:APQ5	Shimmer:APQ11	Shimmer:DDA	NHR	HNR	RPDE	DFA	PPE	motor_UPDRS	total_UPDRS
count	5875.000000	5875.000000	5875.000000	5875.000000	5875.000000	5875.000000	5875.000000	5875.000000	5875.000000	5875.000000	...	5875.000000	5875.000000	5875.000000	5875.000000	5875.000000	5875.000000	5875.000000	5875.000000	5875.000000	5875.000000
mean	64.804936	0.317787	92.863722	0.006154	0.000044	0.002987	0.003277	0.008962	0.034035	0.310960	...	0.020144	0.027481	0.051467	0.032120	21.679495	0.541473	0.653240	0.219589	21.296229	29.018942
std	8.821524	0.465656	53.445602	0.005624	0.000036	0.003124	0.003732	0.009371	0.025835	0.230254	...	0.016664	0.019986	0.039711	0.059692	4.291096	0.100986	0.070902	0.091498	8.129282	10.700283
min	36.000000	0.000000	-4.262500	0.000830	0.000002	0.000330	0.000430	0.000980	0.003060	0.026000	...	0.001940	0.002490	0.004840	0.000286	1.659000	0.151020	0.514040	0.021983	5.037700	7.000000
25%	58.000000	0.000000	46.847500	0.003580	0.000022	0.001580	0.001820	0.004730	0.019120	0.175000	...	0.010790	0.015665	0.027830	0.010955	19.406000	0.469785	0.596180	0.156340	15.000000	21.371000
50%	65.000000	0.000000	91.523000	0.004900	0.000034	0.002250	0.002490	0.006750	0.027510	0.253000	...	0.015940	0.022710	0.041110	0.018448	21.920000	0.542250	0.643600	0.205500	20.871000	27.576000
75%	72.000000	1.000000	138.445000	0.006800	0.000053	0.003290	0.003460	0.009870	0.039750	0.365000	...	0.023755	0.032715	0.061735	0.031463	24.444000	0.614045	0.711335	0.264490	27.596500	36.399000
max	85.000000	1.000000	215.490000	0.099990	0.000446	0.057540	0.069560	0.172630	0.268630	2.107000	...	0.167020	0.275460	0.488020	0.748260	37.875000	0.966080	0.865600	0.731730	39.511000	54.992000

	MAE	RMSE	R2	time
RandomForestRegressor	0.719561	1.589440	0.977202	6.863475
GradientBoostingRegressor	1.035064	1.802294	0.970687	6.697103
BaggingRegressor	0.717353	1.584962	0.977330	7.152953

	MAE	RMSE	R2	Train time (s)
AdaBoostRegressor	8,2904	9,6651	0,1570	0,1004
VotingRegressor	3,0640	3,7446	0,8735	9,7425
HistGradientBoostingRegressor	1,2451	2,0134	0,9634	1,1233
GradientBoostingRegressor	1,0351	1,8023	0,9707	6,6971
XGBRegressor	1,0424	1,7563	0,9722	0,3906
RandomForestRegressor	0,7196	1,5894	0,9772	6,8635
BaggingRegressor	0,7174	1,5850	0,9773	7,1530
StackingRegressor	0,7880	1,5414	0,9786	91,4056
TabPFNRegressor	0,2060	0,7836	0,9945	3,0418

*Курсовая работа*¶

*Сравнение ансамблевых методов решения классических ml задач с методом TabPFN*¶

Введение¶

Актуальность темы¶

Цель и задачи исследования¶

Объект и предмет исследования¶

Методы исследования¶

Структура работы¶

Практическая значимость¶

Соответствие требованиям¶

Импортирование библиотек и функций¶

Локализация численных выводов¶

Глава 1. Теоретическая справка алгоритма TabPFN¶

1.1. История создания и контекст¶

1.1.1. Проблемы традиционных методов¶

1.1.2. Возникновение идеи¶

1.2. Архитектура TabPFN¶

1.2.1. Кодирование признаков¶

1.2.2. Механизм внимания¶

1.2.3. Объединение train и test данных¶

1.3. Процесс предобучения¶

1.3.1. Генерация синтетических данных¶

1.3.2. Байесовский идеальный предиктор¶

1.3.3. Обучение без обратного распространения¶

1.4. Принципы работы¶

1.4.1. Предсказание без настройки¶

1.4.2. Интеграция с библиотеками¶

1.4.3. Обработка пропусков¶

1.5. Преимущества и ограничения¶

1.5.1. Преимущества¶

1.5.2. Ограничения¶

1.5.3. Преимущества перед ансамблевыми методами¶

1.6. Выводы¶

Глава 2: Данные для классификации¶

2.1 Загрузка данных¶

2.2 Проверка структуры данных¶

2.2.1 Показать первые строки¶

2.2.2 Информация о типах данных и пропусках¶

2.2.3 Распределение классов¶

2.3 Предобработка данных¶

2.3.1 Кодирование целевой переменной¶

2.3.2 Нормализация признаков¶

2.4 Разделение данных на обучающую и тестовую выборки¶

2.5 Визуализация данных¶

2.5.1 seaborn.pairplot для анализа взаимосвязей признаков¶

2.5.2 Гистограммы распределения признаков¶

2.5.3 Матрица корреляций¶

Глава 3. Экспериментальное исследование¶

3.0 Подготовка к сравнению моделей¶

3.1. Подготовка к сравнению моделей¶

3.2. Модель 1: Случайный лес (Random Forest)¶

3.2.1. Теоретическое описание¶

3.2.2. Подбор гиперпараметров¶

3.2.3. Оценка производительности¶

3.2.4. Визуализация важности признаков¶

3.3. Модель 2: Градиентный бустинг (Gradient Boosting)¶

3.3.1. Теоретическое описание¶

3.3.2. Подбор гиперпараметров¶

3.3.3. Оценка производительности¶

3.3.4. Визуализация важности признаков¶

3.4. Модель 3: Бэггинг (Bagging)¶

3.4.1. Теоретическое описание¶

3.4.2. Подбор гиперпараметров¶

3.4.3. Оценка производительности¶

3.5. Модель 4: Адаптивный бустинг (AdaBoost)¶

3.5.1. Теоретическое описание¶

3.5.2. Подбор гиперпараметров¶

3.5.3. Оценка производительности¶

3.5.4. Визуализация важности признаков¶

3.6. Модель 5: XGBoost¶

3.6.1. Теоретическое описание¶

3.6.1.1. Алгоритм работы¶

3.6.1.2. Преимущества и недостатки¶

3.6.2. Подбор гиперпараметров¶

3.6.2.1. Этапы настройки¶

3.6.2.2. Анализ влияния параметров¶

3.6.3. Оценка производительности¶

3.6.4. Визуализация важности признаков¶

3.6.4.1. Интерпретация¶

3.6.4.2. Сравнение с другими моделями¶

Курсовая работа¶

Сравнение ансамблевых методов решения классических ml задач с методом TabPFN¶

2.5.1 `seaborn.pairplot` для анализа взаимосвязей признаков¶